# -*- coding: utf-8 -*-
import traceback, re
import sqlite3
import cStringIO    as StringIO

from    pymongo     import  Connection
from    lxml        import  etree  
from    termcolor   import  cprint

#MONGO_SERVER        =   '27.0.12.106'
MONGO_SERVER        =   'beta.mana.vn'
MONGO_PORT          =   27017
DATABASE            =   'dictionary'
stringify           =   etree.XPath("string()")

def processContent(a):
    try:
        if a==None or a=='': return
        a = a.replace('<li>', '')
        a = a.replace('</li>', '')
        p   =   re.compile(r'<span.*?>')
        a = a.replace('</span>', '')
        a = a.replace(' class="aexample"', '')
        return  p.sub('', a)
    except:
        traceback.print_exc()

def process():
    try:
        conn_sqlite        =   sqlite3.connect('/home/daudm/anh_viet.db')
        #conn_sqlite        =   sqlite3.connect('/allfile/data/programming/anh_viet.db')
        c                  =   conn_sqlite.cursor()
        conn_mongo         =   Connection(MONGO_SERVER, MONGO_PORT)
        db                 =   conn_mongo[DATABASE]
        collection         =   db['anh_viet']
        c.execute('select * from anh_viet')
        docs = []
        for row in c: 
            content = []
            text = row[2]
            if type(text).__name__ == 'unicode': text = text.encode('utf-8')
            tree    =   etree.parse(StringIO.StringIO(text), etree.HTMLParser(encoding='utf-8'))
            
            titleNode   = tree.xpath('//span[@class="title"]')
            pronunciation = ''
            if len(titleNode) > 0:
                word = stringify(titleNode[0])
                preg = re.compile(r'(/.+/)')
                m    = preg.search(word)
                if m: pronunciation = m.group(1)
            listNode    =   tree.xpath('//html/body/*')
            text     =   ''
            for node in listNode: 
                if node.tag == 'br': continue
                # Nếu là tag ul
                if node.tag == 'ul': 
                    tmp1 = []; tmp2 = []; tmp3 = []
                    childNode   =   node.xpath('./*')
                    # Duyệt qua các node
                    for child in childNode:
                        # Nếu gặp node ul trong li thì lấy nội dung rồi remove
                        ulNode      =   child.xpath('./ul')
                        if len(ulNode) > 0: 
                            tmp2 = []
                            liNode  =   ulNode[0].xpath('./li')
                            if len(liNode) > 0:
                                for li in liNode:
                                    # Nếu gặp node ul thì lấy nội dung rồi remove
                                    ullNode     =   li.xpath('./ul')
                                    tmp3 = []
                                    if len(ullNode) > 0:
                                        liiNode = ullNode[0].xpath('./li')
                                        for lii in liiNode:
                                            #cprint(stringify(lii), 'yellow')
                                            chtml   =   etree.tounicode(lii, method='html')    
                                            cprint('####' + chtml, 'green')
                                            print processContent(chtml)
                                            tmp3.append(chtml)
                                        ullNode[0].getparent().remove(ullNode[0])

                                    #cprint(stringify(li), 'yellow')
                                    chtml   =   etree.tounicode(li, method='html')    
                                    chtml =  processContent(chtml)
                                    cprint('###' + chtml, 'white')
                                    tmp2.append({'text': chtml, 'content': tmp3})
                                    print '-----------'
                            ulNode[0].getparent().remove(ulNode[0])
                            text2   =   etree.tounicode(child, method='html')    
                            text2   =   processContent(text2)
                            tmp1.append({'text': text2, 'content': tmp2})
                            cprint('##' + text2, 'yellow')
                        else:
                            text1   =   etree.tounicode(child, method='html')    
                            text1   =   processContent(text1)
                            tmp1.append({'text': text1, 'content': []})
                            print child.tag, '#' + text1
                            print '----------------'

                    content.append({'text': text, 'content': tmp1})
                else:
                    text = stringify(node)
                    if type(text).__name__ == 'unicode': text = text.encode('utf-8')
                    print node.tag, text
                print '---------------------'
            doc = {'_id': row[0], 'word': row[1], 'pronunciation': pronunciation, 'content': content}
            docs.append(doc)
            print '###############################################'

        for doc in docs: 
            result      =   collection.find_one({'_id': doc['_id']})
            if result==None: 
              collection.save(doc)
            else: 
              cprint('Từ đã tồn tại trong cơ sở dữ liệu', 'red')
    except:
        traceback.print_exc()

if __name__ == '__main__':
    try:
        process()
    except:
        traceback.print_exc()
